package pipeline;

import java.io.File;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.Hashtable;
import java.util.Map.Entry;

import setting.BioNLPSettings;
import util.io.FileUtil;
import util.ling.Porter;
import abs.syn.DependencyPath;
import abs.syn.Parse;
import classifier.SVMWrapper;
import def.AbstractText;
import def.ArgType;
import def.Argument;
import def.BioNLPDocument;
import def.Entity;
import def.Event;
import def.Protein;
import def.TSpan;
import def.TextType;
import exp.Count;

public class EdgeDetectionSVM extends EdgeDetection {

	public EdgeDetectionSVM() {
		//		init();
	}
	@Override
	public void init(){
		String[] lines = FileUtil.getTextFromFile(BioNLPSettings.getValue("edgeFeatureMap")).split("\n");
		if (lines.length!=1){
			System.out.println("reading ... "+lines.length +" feature map");
			for (String line: lines){
				String[] pair = line.split("\t");
				str2id.put(pair[0], Integer.parseInt(pair[1]));
			}
		}

	}

	@Override 
	public Hashtable<TSpan, Event> build(BioNLPDocument doc, ArrayList<TSpan> triggers, Hashtable<String, Protein> corefProteins){
		Hashtable<TSpan, Event> predEvents = new Hashtable<TSpan, Event>();
		//		Collection<Argument> proteins = doc.tspans.values();
		ArrayList<Protein> proteins = new ArrayList<Protein>(doc.proteins);

		// remove "PROTEIN (PROTEIN)"
		for (Protein p : corefProteins.values()){
			proteins.remove(p);
		}

		System.err.println("*********[EDGE]"+doc.fileId+"**********");
		SVMWrapper svm = new SVMWrapper();
		float linkThresh = BioNLPSettings.getFloatValue("LINK_THRESHOLD"), roleThresh = BioNLPSettings.getFloatValue("ROLE_THRESHOLD");
		int rec = 0, all = 0;
		// recall with completed jobs
		int comprec = 0, compall = 0;
		ArrayList<Hashtable<String, Integer>> examples = new ArrayList<Hashtable<String, Integer>>();
		for (int i =0 ; i < doc.aText.lines.length; i++){
			//			HashSet<Event> curEvents = new HashSet<Event>();
			//			System.err.println("====LINE: "+doc.aText.lines[i]+"========");
			// 3a. for each line of text
			// 3b. for each pair of trigger and protein, if it fits a pattern, add it in.
//			Parse parse = doc.parses[i];
			
			//			int regNum = 0;

			//			for (TSpan trigger: triggers){
			for (int k = 0; k < triggers.size(); k++){
				TSpan trigger = triggers.get(k);
				// restrict to the current line
				if (doc.aText.getSentenceIdByPos(trigger.startIdx)!=i) 
					continue;
				//				if (td.parentDictionary.contains(trigger.text.toLowerCase())){
				//					regNum++;
				//				}
//				Event e = null;
				
				// could be protein
				//				for (Argument protein: proteins){
				for (int j = 0; j < proteins.size(); j++){
					Argument protein = proteins.get(j);
					// restrict to the current line
					if (doc.aText.getSentenceIdByPos(protein.tspan.startIdx)!=i) 
						continue;
					
					Hashtable<String, Integer> example = extractFeature(doc, trigger, protein.tspan, i);
					examples.add(example);
				}

				// *** if not in regulation dictionary, then continue *** 
				//				if (!td.parentDictionary.contains(trigger.text.toLowerCase()))
				//					continue;

				// or could be another trigger
				//				for (TSpan anotherTrigger: triggers){
				for (int j = 0; j < triggers.size(); j++){
					TSpan anotherTrigger = triggers.get(j);
					if (anotherTrigger == trigger) 
						continue;
					// restrict to the current line
					if (doc.aText.getSentenceIdByPos(anotherTrigger.startIdx)!=i) 
						continue;
					Hashtable<String, Integer> example = extractFeature(doc, trigger, anotherTrigger, i);
					examples.add(example);
				}
			}
		}
		// ***SVM for both proteins and triggers***
		StringBuilder sb = new StringBuilder();
		for (int j = 0; j < examples.size(); j++){
			//					Argument protein = proteins.get(j);
			//					if (doc.aText.getSentenceIdByPos(protein.tspan.startIdx)!=i) 
			//						continue;
			Hashtable<String, Integer> example = examples.get(j);
			ArrayList<Integer> features = new ArrayList<Integer>();
			for (String key: example.keySet()){
				if (key.startsWith("CLASS"))
				{
					System.err.println("[EDGEDETECTION_SVM] found CLASS feature !! "+key);
					continue;
				}
				else{
					if (str2id.containsKey(key)){
						features.add(str2id.get(key));
					}
					else{
						System.err.println("[EDGE_SVM] missing feature id for {"+key+"}");
					}
				}
			}
			Collections.sort(features);
			sb.append("1");
			for (int jj = 0; jj < features.size(); jj++){
				sb.append(" "+features.get(jj)+":1");
			}
			sb.append("\n");
		}
		FileUtil.writeTextToFile(sb.toString(), "edge_example.svm");
		svm.test("edge_example.svm", BioNLPSettings.getValue("EDGE_MODEL"), "edge_example_prediction");
		svm.test("edge_example.svm", BioNLPSettings.getValue("ROLE_MODEL"), "edge_role_prediction");

		// get predictions
		String[] lscores = FileUtil.getTextFromFile("edge_example_prediction").split("\n"),
		rscores = FileUtil.getTextFromFile("edge_role_prediction").split("\n");
		int idx = 0;
		for (int i =0 ; i < doc.aText.lines.length; i++){
			HashSet<Event> curEvents = new HashSet<Event>();
			System.err.println("====LINE: "+doc.aText.lines[i]+"========");
			// 3a. for each line of text
			// 3b. for each pair of trigger and protein, if it fits a pattern, add it in.
			Parse parse = doc.parses[i];
			int numTrigger =  0, numProtein = 0;
			//			int regNum = 0;

			//				for (TSpan trigger: triggers){
			for (int k = 0; k < triggers.size(); k++){
				TSpan trigger = triggers.get(k);
				// restrict to the current line
				if (doc.aText.getSentenceIdByPos(trigger.startIdx)!=i) 
					continue;
				//				if (td.parentDictionary.contains(trigger.text.toLowerCase())){
				//					regNum++;
				//				}
				Event e = null;
				numTrigger ++;
				for (int j = 0; j < proteins.size(); j++){
					Argument protein = proteins.get(j);
					if (doc.aText.getSentenceIdByPos(protein.tspan.startIdx)!=i) 
						continue;
					numProtein++;
					float s = Float.parseFloat(lscores[idx]);
					if (s > linkThresh){
						// further decide which role

						float s2 = Float.parseFloat(rscores[idx]);

						if (e==null){
							if ( !predEvents.containsKey(trigger)){
								e= new Event();
								e.tspan = trigger;
								predEvents.put(trigger, e);
								curEvents.add(e);
							}
							else{
								e = predEvents.get(trigger);
							}
						}
						e.args.add(protein);
						if (s2 > roleThresh){
							// THEME
							e.types.add(ArgType.Theme);
							System.err.println(trigger.text+"; "+protein.tspan.text+", trigger-protein LINK THEME lscore="+s+", rscore="+s2);
						}
						else{
							// CAUSE
							e.types.add(ArgType.Cause);
							System.err.println(trigger.text+"; "+protein.tspan.text+", trigger-protein LINK CAUSE lscore="+s+", rscore="+s2);
						}
					}
					else{
						System.err.println(trigger.text+"; "+protein.tspan.text+", trigger-protein NO LINK  lscore="+s);
						// no link
					}
					idx++;
				}

				for(int j = 0; j < triggers.size(); j++){

					TSpan anotherTrigger = triggers.get(j);
					if (anotherTrigger == trigger) 
						continue;
					// restrict to the current line
					if (doc.aText.getSentenceIdByPos(anotherTrigger.startIdx)!=i) 
						continue;
					float s = Float.parseFloat(lscores[idx]);
					if (s > linkThresh){
						// further decide which role
						float s2 = Float.parseFloat(rscores[idx]);
						if (e==null){
							if ( !predEvents.containsKey(trigger)){
								e= new Event();
								e.tspan = trigger;
								predEvents.put(trigger, e);
								curEvents.add(e);
							}
							else{
								e = predEvents.get(trigger);
							}
						}
						Event anotherEvent = null;
						if (!predEvents.containsKey(anotherTrigger)){
							anotherEvent =  new Event();
							anotherEvent.tspan = anotherTrigger;
							predEvents.put(anotherTrigger, anotherEvent);
							curEvents.add(anotherEvent);
						}
						else{
							anotherEvent = predEvents.get(anotherTrigger);
						}
						// check if adding will cause loop
						// FIXME the order of adding events matters 
						if (!anotherEvent.hasChild( e)){
							e.args.add(anotherEvent);
							if (s2 > roleThresh){
								// THEME
								System.err.println(trigger.text+"; "+anotherTrigger.text+", trigger-trigger LINK THEME lscore="+s+", rscore="+s2);
								e.types.add(ArgType.Theme);
							}
							else{
								// CAUSE
								System.err.println(trigger.text+"; "+anotherTrigger.text+", trigger-trigger LINK CAUSE lscore="+s+", rscore="+s2);
								e.types.add(ArgType.Cause);
							}
						}
						else{
							System.err.println(trigger.text+"; "+anotherTrigger.text+", trigger-trigger NO LINK TO AVOID LOOPS lscore="+s);
						}
					}
					else{
						// no link
						System.err.println(trigger.text+"; "+anotherTrigger.text+", trigger-trigger NO LINK lscore="+s);
					}
					idx++;
				}
				
			}
			if (numProtein!=0)
				System.err.println("[STAT]\t"+numTrigger+"\t"+numProtein/*+"\t"+regNum*/);
			if (numProtein> 600){
				System.err.println("[TEXT] "+doc.fileId+" \t"+doc.aText.lines[i]);
			}
		}
		// ***END SVM***

		return predEvents;
	}



	public ArrayList<Hashtable<String, Integer>> examples = new ArrayList<Hashtable<String, Integer>>();
	public ArrayList<String> pairIds = new ArrayList<String>();
	public Hashtable<String, Integer> str2id = new Hashtable<String, Integer>();

	@Override
	public Count processDocument(String path, String filePrefix){
		Count c = new Count();
		System.err.println("[EDGE_LEARN]"+filePrefix);
		BioNLPDocument doc = new BioNLPDocument(path, filePrefix);
		if (doc.tspans.size() == 0){
			System.out.println(filePrefix + " skipped");
			return c;
		}

		if (!doc.valid)
			return c;

		/*// get all pairs of tspans in the same sentence
		// including detected triggers.
		ArrayList<TSpan> triggers = td.detectTriggers(doc);
		int start = 100;
		for(TSpan pred : triggers){
			// if pred matches any of the gold tspan
			TSpan match = null;
			
			for (String key: doc.tspans.keySet()){
				if (!key.startsWith("T"))
					continue;
				if (doc.tspans.get(key) instanceof Protein || doc.tspans.get(key) instanceof Entity) 
					continue;
				TSpan target = doc.tspans.get(key).tspan;
				if (target.startIdx <= pred.startIdx && pred.endIdx <= target.endIdx){
					match = target;
					break;
				}
			}
			
			if(match!=null){
				// replace it with detected one
				// FIXME be careful with multiple spans, maybe two predictions match the same one
				TSpan target = match;
				target.startIdx = pred.startIdx;
				target.endIdx = pred.endIdx;
				target.text = pred.text;
			}
			else{
				// add it in doc.tspans with generated tid, e.g. T100
				Event e = new Event();
				e.setTspan(pred);
				pred.tid = "T"+start;
				start++;
				doc.tspans.put(pred.tid, e);
				System.err.println("[EDGE_TRIGGER] add "+pred);
			}
		}
		// **END predicted triggers***
*/		
		HashSet<String> candidates = new HashSet<String>();

		for (int i = 0; i < doc.parses.length; i++){
			for (String key: doc.tspans.keySet()){
				if (!key.startsWith("T"))
					continue;
				if (doc.tspans.get(key) instanceof Protein || doc.tspans.get(key) instanceof Entity) 
					continue;
				int sentid = doc.getSentenceIdByPos(doc.tspans.get(key).tspan.startIdx);
				for (String key2 :  doc.tspans.keySet()){
					if (key == key2)  continue;
					if (!key2.startsWith("T"))continue;
					if (doc.tspans.get(key2) instanceof Entity) continue;
					if (doc.getSentenceIdByPos(doc.tspans.get(key2).tspan.startIdx)== sentid){
						candidates.add(key+"\t"+key2);
					}
				}
			}
		}
		System.err.println("[EDGE_SVM] |candidates| = "+candidates.size());
		
		// 
		
		//		for (String str: candidates){
		//			System.err.println("[CANDIDATES] "+str);
		//		}

		// for each true link, label them positive
		// remove them from candidates
		System.err.println("[EDGE_SVM] including positives ");
		AbstractText text = (AbstractText)doc.text;
		for (Event e: doc.events.values()){

			int eventSentIdx = text.getSentenceIdByPos(e.tspan.startIdx);
			for (int i = 0; i < e.args.size(); i++){
				int argSentIdx = text.getSentenceIdByPos(e.args.get(i).tspan.startIdx);
				if (eventSentIdx != argSentIdx){
					System.out.println("processDocument@EventDepTemplate : skipping pair (not in the same sentence) for {"+e+" @@ "+e.args.get(i)+"}");
				}
				else{
					String pair = e.tspan.tid + "\t" + e.args.get(i).tspan.tid;
					//					System.err.println("[CANDIDATES] "+pair);
					if (!candidates.contains(pair))
						continue;
					candidates.remove(pair);
					Hashtable<String, Integer> example = extractFeature(doc, e,e.args.get(i), eventSentIdx );
					example.put("CLASS_"+e.types.get(i).toString().replaceAll("\\d+", ""), 1);
					examples.add(example);
					pairIds.add(doc.fileId+"\t"+e.getId() + "\t" + e.args.get(i).getId());
				}
			}
		}

		// for the rest in the candidate pair, label them false;
		System.err.println("[EDGE_SVM] including negatives");
		for (String str: candidates){
			//			System.err.println("[CANDIDATES] "+str);
			String[] pair = str.split("\t");
			
			//FIXME doc.tspans -> candidates
			Argument arg1 = doc.tspans.get(pair[0]), arg2 = doc.tspans.get(pair[1]);
			int eventSentIdx = text.getSentenceIdByPos(arg1.tspan.startIdx);
			Hashtable<String, Integer> example = extractFeature(doc, arg1, arg2, eventSentIdx );
			example.put("CLASS_NA", 1);
			examples.add(example);
			pairIds.add(doc.fileId+"\t"+arg1.getId() + "\t" + arg2.getId());
		}

		return c;
	}
	public static Porter stemmer = new Porter();

	private Hashtable<String, Integer> extractFeature(BioNLPDocument doc,
			Argument e, Argument argument, int sentId) {
		Hashtable<String, Integer> example = new Hashtable<String, Integer>();

		//		trigger, pos, stem, word, isHead?
		Parse parse = doc.parses[sentId];
		int offset = doc.aText.sentStartPos[sentId];
		int triggerWord = parse.getWord(e.tspan.startIdx - offset);
		int argWord =parse.getWord(argument.tspan.startIdx - offset);
		int triggerHead = parse.getHead(triggerWord);
		int argHead = parse.getHead(argWord);
		String tWord = null, tPos = null;
		if (triggerHead!=-1 && e.tspan.startIdx <= parse.startPos[triggerHead] && parse.startPos[triggerHead] <= e.tspan.endIdx){
			tWord = parse.words[triggerHead].toLowerCase();
			tPos = parse.leaves.get(triggerHead).parent.getName();
			example.put("TRG_WORD_"+tWord+"_"+tPos, 1);
			example.put("TRG_STEM_"+stemmer.stripAffixes(tWord)+"_"+tPos.substring(0,1), 1);
			example.put("TRG_HAS_HEAD", 1);
		}else{
			tWord = parse.words[triggerWord].toLowerCase();
			tPos = parse.leaves.get(triggerWord).parent.getName();
			example.put("TRG_WORD_"+tWord+"_"+tPos, 1);
			example.put("TRG_STEM_"+stemmer.stripAffixes(tWord)+"_"+tPos.substring(0,1), 1);
			example.put("TRG_HAS_NO_HEAD", 1);
		}
		// arg, word, pos, stem/ protein?, isHead?
		String aWord = null; 
		String aPos = null;
		if (argument instanceof Protein){
			example.put("ARG_PROTEIN", 1);
			aWord = "PROTEIN";
			aPos = "N";
		}
		else if (argument instanceof Event){
			example.put("ARG_EVENT", 1);
			aWord = argument.tspan.text.toLowerCase();
			aPos = parse.leaves.get(argWord).parent.getName();
		}
		
		if (argHead!=-1 && argument.tspan.startIdx <= parse.startPos[argHead] && parse.startPos[argHead] <= argument.tspan.endIdx){
//			example.put("ARG_HAS_HEAD", 1);
			example.put("ARG_HAS_HEAD_ITSELF", 1);
			if (argHead!=-1){
				example.put("ARG_HAS_HEAD_"+stemmer.stripAffixes(parse.words[argHead].toLowerCase()), 1);
			}
			if (argument instanceof Event){
//				String aWord = parse.words[argHead].toLowerCase();
//				String aPos = parse.leaves.get(argHead).parent.getName();
				example.put("TRG_ARG_"+stemmer.stripAffixes(tWord)+"_"+tPos.substring(0,1)
						+"_"+stemmer.stripAffixes(aWord)+"_"+aPos.substring(0,1), 1);
				example.put("ARG_WORD_"+aWord+"_"+aPos, 1);
				example.put("ARG_STEM_"+stemmer.stripAffixes(aWord)+"_"+aPos.substring(0,1), 1);
			}else{
				example.put("TRG_ARG_"+stemmer.stripAffixes(tWord)+"_"+tPos.substring(0,1)
						+"_PROTEIN", 1);
//				example.put("ARG_WORD_PROTEIN", 1);
			}
		}
		else{
//			example.put("ARG_HAS_NO_HEAD", 1);
			example.put("ARG_HAS_HEAD_ELSE", 1);
			if (argHead!=-1){
				example.put("ARG_HAS_HEAD_"+stemmer.stripAffixes(parse.words[argHead].toLowerCase()), 1);
			}
			if (argument instanceof Event){
//				String aWord = parse.words[argWord].toLowerCase();
//				String aPos = parse.leaves.get(argWord).parent.getName();
				example.put("TRG_ARG_"+stemmer.stripAffixes(tWord)+"_"+tPos.substring(0,1)
						+"_"+stemmer.stripAffixes(aWord)+"_"+aPos.substring(0,1), 1);
				example.put("ARG_WORD_"+aWord+"_"+aPos, 1);
				example.put("ARG_STEM_"+stemmer.stripAffixes(aWord)+"_"+aPos.substring(0,1), 1);
			}else{
				example.put("TRG_ARG_"+stemmer.stripAffixes(tWord)+"_"+tPos.substring(0,1)
						+"_PROTEIN", 1);
//				example.put("ARG_WORD_PROTEIN", 1);
			}
		}

		// combination of trigger + dep
		// dep
		// directed path
		// undirected path
		// length
		DependencyPath pp = doc.parses[sentId].getShortestPath(e.tspan.startIdx - offset, e.tspan.endIdx - offset, 
				argument.tspan.startIdx - offset, argument.tspan.endIdx - offset);
		//		System.err.println("[EDGE_LEARN]"+doc.fileId + "Event: "+e+", Argument: "+argument);
		//		System.err.println("[EDGE_LEARN]"+doc.fileId + "Text: "+ doc.aText.lines[sentId]);
		//					System.out.println("Found paths : "+list);

		if (pp!=null){
			String p1 = "", p2 = "";
			for (int j = 0; j < pp.deps.size(); j++){
				p1 += pp.deps.get(j).name+(pp.isLeftArgument.get(j)?"+":"-");
				p2 += pp.deps.get(j).name;
				String otherWord = null;
				if (pp.deps.get(j).govIdx == -1 || pp.deps.get(j).depIdx == - 1){
					System.err.println("[DELETEME] "+doc.fileId+"  " + sentId + " "+pp.deps.get(j));
				}
				if (pp.isLeftArgument.get(j))
					otherWord = stemmer.stripAffixes(parse.words[pp.deps.get(j).depIdx-1].toLowerCase());
				else
					otherWord = stemmer.stripAffixes(parse.words[pp.deps.get(j).govIdx-1].toLowerCase());

				example.put("DEP_"+pp.deps.get(j).name, 1);
				example.put("DEP_WORD_"+pp.deps.get(j).name+"_"+otherWord, 1);
				example.put("DEP_TRG_"+stemmer.stripAffixes(tWord)+"_"+tPos.substring(0,1)+"_"+pp.deps.get(j).name, 1);
				if (j == 0){
					example.put("DEP_FIRST_"+pp.deps.get(j).name, 1);
					example.put("DEP_FIRST_TRG_"+stemmer.stripAffixes(tWord)+"_"+tPos.substring(0,1)+"_"+pp.deps.get(j).name, 1);
				}
				if (j == pp.deps.size()-1){
					example.put("DEP_LAST_"+pp.deps.get(j).name, 1);
					example.put("DEP_LAST_TRG_"+stemmer.stripAffixes(tWord)+"_"+tPos.substring(0,1)+"_"+pp.deps.get(j).name, 1);
				}
				example.put("DEP_DIRECT_"+pp.deps.get(j).name+(pp.isLeftArgument.get(j)?"+":"-"), 1);
			}
			//			example.put("DEPPATH_UNDIRECT_"+p2, 1);
			//			example.put("DEPPATH_UNDIRECT_TRG_"+tWord+"_"+tPos+"_"+p2, 1);
			//			example.put("DEPPATH_DIRECT_"+p1, 1);
			//			example.put("DEPPATH_DIRECT_TRG_"+tWord+"_"+tPos+"_"+p1, 1);
			example.put("DEP_LENGTH_"+pp.deps.size(), 1);
		}
		else{
			System.err.println("[EDGE_LEARN] cant find dep path" );
			example.put("DEP_NOT_FOUND", 1);
		}

		// HEAD DEP
		if (argHead!=-1 && triggerHead != -1){
			pp = parse.getShortestPath(triggerHead, argHead);
			if (pp!=null){
			example.put("HEAD_DEP_DIRECT_"+pp.getFeatureString(true),1 );
			example.put("HEAD_DEP_UNDIRECT_"+pp.getFeatureString(false),1 );
			example.put("HEAD_DEP_DIRECT_TRG_ARG_"+pp.getFeatureString(true)+"_"
					+stemmer.stripAffixes(parse.words[triggerHead].toLowerCase())+"_"
					+stemmer.stripAffixes(parse.words[argHead].toLowerCase()), 1);
			}
			else{
				example.put("HEAD_DEP_NOT_FOUND", 1);
			}
		}
		else{
			example.put("HEAD_DEP_NOT_FOUND", 1);
		}
		
		
		return example;
	}

	private Hashtable<String, Integer> extractFeature(BioNLPDocument doc,
			TSpan e, TSpan argument, int sentId) {
		Hashtable<String, Integer> example = new Hashtable<String, Integer>();

		//		trigger, pos, stem, word, isHead?
		Parse parse = doc.parses[sentId];
		int offset = doc.aText.sentStartPos[sentId];
		int triggerWord = parse.getWord(e.startIdx - offset);
		int argWord =parse.getWord(argument.startIdx - offset);
		int triggerHead = parse.getHead(triggerWord);
		int argHead = parse.getHead(argWord);
		String tWord = null, tPos = null;
		if (triggerHead!=-1 && e.startIdx <= parse.startPos[triggerHead] && parse.startPos[triggerHead] <= e.endIdx){
			tWord = e.text.toLowerCase();
			tPos = parse.leaves.get(triggerHead).parent.getName();
			example.put("TRG_WORD_"+tWord+"_"+tPos, 1);
			example.put("TRG_STEM_"+stemmer.stripAffixes(tWord)+"_"+tPos.substring(0,1), 1);
			example.put("TRG_HAS_HEAD", 1);
		}else{
			tWord = e.text.toLowerCase();
			tPos = parse.leaves.get(triggerWord).parent.getName();
			example.put("TRG_WORD_"+tWord+"_"+tPos, 1);
			example.put("TRG_STEM_"+stemmer.stripAffixes(tWord)+"_"+tPos.substring(0,1), 1);
			example.put("TRG_HAS_NO_HEAD", 1);
		}
		// arg, word, pos, stem/ protein?, isHead?
		if (argument.textType == TextType.Protein){
			example.put("ARG_PROTEIN", 1);
		}
		else {
			example.put("ARG_EVENT", 1);
		}
		String aWord = null; 
		String aPos = null;
		if (argument.textType != TextType.Protein){
			// event trigger
			aWord = argument.text.toLowerCase();
			aPos = parse.leaves.get(argWord).parent.getName();
		}else {
			// protein
			aWord = "PROTEIN";
			aPos = "N";
		}
		if (argHead!=-1 && argument.startIdx <= parse.startPos[argHead] && parse.startPos[argHead] <= argument.endIdx){
			example.put("ARG_HAS_HEAD_ITSELF", 1);
			if (argHead!=-1){
				example.put("ARG_HAS_HEAD_"+stemmer.stripAffixes(parse.words[argHead].toLowerCase()), 1);
			}
			if (argument.textType != TextType.Protein){
				example.put("TRG_ARG_"+stemmer.stripAffixes(tWord)+"_"+tPos.substring(0,1)
						+"_"+stemmer.stripAffixes(aWord)+"_"+aPos.substring(0,1), 1);
				example.put("ARG_WORD_"+aWord+"_"+aPos, 1);
				example.put("ARG_STEM_"+stemmer.stripAffixes(aWord)+"_"+aPos.substring(0,1), 1);
			}
			else{
				example.put("TRG_ARG_"+stemmer.stripAffixes(tWord)+"_"+tPos.substring(0,1)
						+"_PROTEIN", 1);
//				example.put("ARG_WORD_PROTEIN", 1);
			}
		}
		else{
			example.put("ARG_HAS_HEAD_ELSE", 1);
			if (argHead!=-1){
				example.put("ARG_HAS_HEAD_"+stemmer.stripAffixes(parse.words[argHead].toLowerCase()), 1);
			}
			if (argument.textType != TextType.Protein){
//				String aWord = argument.text.toLowerCase();//parse.words[argWord].toLowerCase();
//				String aPos = parse.leaves.get(argWord).parent.getName();
				example.put("TRG_ARG_"+stemmer.stripAffixes(tWord)+"_"+tPos.substring(0,1)
						+"_"+stemmer.stripAffixes(aWord)+"_"+aPos.substring(0,1), 1);
				example.put("ARG_WORD_"+aWord+"_"+aPos, 1);
				example.put("ARG_STEM_"+stemmer.stripAffixes(aWord)+"_"+aPos.substring(0,1), 1);
			}
			else{
				example.put("TRG_ARG_"+stemmer.stripAffixes(tWord)+"_"+tPos.substring(0,1)
						+"_PROTEIN", 1);
//				example.put("ARG_WORD_PROTEIN", 1);
			}
		}

		// combination of trigger + dep
		// dep
		// directed path
		// undirected path
		// length
		DependencyPath pp = doc.parses[sentId].getShortestPath(e.startIdx - offset, e.endIdx - offset, 
				argument.startIdx - offset, argument.endIdx - offset);
		//		System.err.println("[EDGE_LEARN]"+doc.fileId + "Event: "+e+", Argument: "+argument);
		//		System.err.println("[EDGE_LEARN]"+doc.fileId + "Text: "+ doc.aText.lines[sentId]);
		//					System.out.println("Found paths : "+list);

		if (pp!=null){
			String p1 = "", p2 = "";
			for (int j = 0; j < pp.deps.size(); j++){
				p1 += pp.deps.get(j).name+(pp.isLeftArgument.get(j)?"+":"-");
				p2 += pp.deps.get(j).name;
				String otherWord = null;
				if (pp.deps.get(j).govIdx == -1 || pp.deps.get(j).depIdx == - 1){
					System.err.println("[DELETEME] "+doc.fileId+"  " + sentId + " "+pp.deps.get(j));
				}
				if (pp.isLeftArgument.get(j))
					otherWord = stemmer.stripAffixes(parse.words[pp.deps.get(j).depIdx-1].toLowerCase());
				else
					otherWord = stemmer.stripAffixes(parse.words[pp.deps.get(j).govIdx-1].toLowerCase());

				example.put("DEP_"+pp.deps.get(j).name, 1);
				example.put("DEP_WORD_"+pp.deps.get(j).name+"_"+otherWord, 1);
				example.put("DEP_TRG_"+stemmer.stripAffixes(tWord)+"_"+tPos.substring(0,1)+"_"+pp.deps.get(j).name, 1);
				if (j == 0){
					example.put("DEP_FIRST_"+pp.deps.get(j).name, 1);
					example.put("DEP_FIRST_TRG_"+stemmer.stripAffixes(tWord)+"_"+tPos.substring(0,1)+"_"+pp.deps.get(j).name, 1);
				}
				if (j == pp.deps.size()-1){
					example.put("DEP_LAST_"+pp.deps.get(j).name, 1);
					example.put("DEP_LAST_TRG_"+stemmer.stripAffixes(tWord)+"_"+tPos.substring(0,1)+"_"+pp.deps.get(j).name, 1);
				}
				example.put("DEP_DIRECT_"+pp.deps.get(j).name+(pp.isLeftArgument.get(j)?"+":"-"), 1);
			}
			//			example.put("DEPPATH_UNDIRECT_"+p2, 1);
			//			example.put("DEPPATH_UNDIRECT_TRG_"+tWord+"_"+tPos+"_"+p2, 1);
			//			example.put("DEPPATH_DIRECT_"+p1, 1);
			//			example.put("DEPPATH_DIRECT_TRG_"+tWord+"_"+tPos+"_"+p1, 1);
			example.put("DEP_LENGTH_"+pp.deps.size(), 1);
		}
		else{
			System.err.println("[EDGE_LEARN] cant find dep path" );
			example.put("DEP_NOT_FOUND", 1);
		}

		// HEAD DEP
		if (argHead!=-1 && triggerHead != -1){
			pp = parse.getShortestPath(triggerHead, argHead);
			if (pp != null){
			example.put("HEAD_DEP_DIRECT_"+pp.getFeatureString(true),1 );
			example.put("HEAD_DEP_UNDIRECT_"+pp.getFeatureString(false),1 );
			example.put("HEAD_DEP_DIRECT_TRG_ARG_"+pp.getFeatureString(true)+"_"
					+stemmer.stripAffixes(parse.words[triggerHead].toLowerCase())+"_"
					+stemmer.stripAffixes(parse.words[argHead].toLowerCase()), 1);
			}
			else{
				example.put("HEAD_DEP_NOT_FOUND", 1);
			}
		}
		else{
			example.put("HEAD_DEP_NOT_FOUND", 1);
		}
		
		
		return example;
	}

	@Override
	public void learn(){
		Count sum = new Count();
		String path = BioNLPSettings.getValue("dataPath");
		//			BioNLPSettings.parsePath = BioNLPSettings.trainParsePath;

		// labeled data
		System.err.println("[EDGE_SVM_LEARN] training...");
		for (String filename :  new File(path).list()){
			//						String filename = "9825820.txt";
			if (filename.endsWith(".txt")){
				//					if (r.nextDouble()> 0.1)
				//						continue;
				String filePrefix = filename.replace(".txt", "");
				System.out.println(filename+ " to be processed");
				sum.add(processDocument(path, filePrefix));
			}
		}
		int trainSize = examples.size();

		path = "data/BioNLP09/dev/";
		String parsePath = "bionlp09_shared_task_devel_analyses_rev2/biomodel";
		BioNLPSettings.setValue("dataPath", path);
		BioNLPSettings.setValue("parsePath", parsePath);
		System.err.println("[EDGE_SVM_LEARN] dev...");
		for (String filename :  new File(path).list()){
			//						String filename = "9825820.txt";
			if (filename.endsWith(".txt")){
				//					if (r.nextDouble()> 0.1)
				//						continue;
				String filePrefix = filename.replace(".txt", "");
				System.out.println(filename+ " to be processed");
				sum.add(processDocument(path, filePrefix));
			}
		}
		System.err.println("encoding...");
		StringBuilder sb = new StringBuilder();	
		if(!new File(BioNLPSettings.getValue("edgeFeatureMap")).exists()){
			for (Hashtable<String ,Integer> ex: examples){
				for (String key: ex.keySet()){
					if (!str2id.containsKey(key)){
						str2id.put(key, str2id.size()+1);
					}
				}
			}
			for (Entry<String, Integer> entry: str2id.entrySet()){
				sb.append(entry.getKey()+"\t"+entry.getValue()+"\n");
			}
			FileUtil.writeTextToFile(sb.toString(), BioNLPSettings.getValue("edgeFeatureMap"));
		}
		else{
			System.err.println("encoding...skipped");
			init();
		}

		System.err.println("printing train...");
		// prepare link svm
		sb = new StringBuilder();
		ArrayList<Integer> labels = new ArrayList<Integer>();
		ArrayList<String> featureStrings = new ArrayList<String>();
		for (int i = 0; i < examples.size(); i++){
			ArrayList<Integer> features = new ArrayList<Integer>();
			int className = 0;
			for (String key: examples.get(i).keySet()){
				if (key.startsWith("CLASS"))
				{
					if (key.endsWith("NA")){
						className = -1;
						labels.add(1);
					}
					else if (key.endsWith("Theme")){
						className = 1;
						labels.add(2);
					}
					else if (key.endsWith("Cause")){
						className = 1;
						labels.add(3);
					}
					else{
						System.err.println("[EDGE_SVM_PRINT] found class key: "+key);
					}
				}
				else{
					if (str2id.containsKey(key)){
						features.add(str2id.get(key));
					}
					else{
						System.err.println("[EDGE_SVM] missing feature id for {"+key+"}");
					}
				}
			}
			Collections.sort(features);
			sb.append(""+className);
			StringBuilder lineStr = new StringBuilder(); 
			for (int j = 0; j < features.size(); j++){
				sb.append(" "+features.get(j)+":1");
				lineStr.append(" "+features.get(j)+":1");
			}
			sb.append("\n");
			featureStrings.add(lineStr.toString());
			if (i == trainSize - 1){
				FileUtil.writeTextToFile(sb.toString(), "train.edge.svm");
				sb = new StringBuilder();
				System.err.println("printing test...");
			}
		}
		FileUtil.writeTextToFile(sb.toString(), "test.edge.svm");

		// classify if there is a link or not
		SVMWrapper svm = new SVMWrapper();
		System.err.println("[EDGE_LEARN] training edge model");
		if (!new File(BioNLPSettings.getValue("EDGE_MODEL")).exists())
			svm.train("train.edge.svm", BioNLPSettings.getValue("EDGE_MODEL")/*"svm_link_model"*/);
		else{
			System.err.println("[EDGE_LEARN] training edge model skipped");
		}
		System.err.println("[EDGE_LEARN] testing edge model");
		svm.test("test.edge.svm", BioNLPSettings.getValue("EDGE_MODEL"), "svm_test_edge_predictions");

		// prepare role svm
		sb = new StringBuilder();
		for (int i = 0; i < trainSize; i++){
			if (labels.get(i) >1 ){
				sb.append(labels.get(i) == 2?1:-1);
				sb.append(featureStrings.get(i));
				sb.append("\n");
			}
		}
		FileUtil.writeTextToFile(sb.toString(), "train.role.svm");
		sb = new StringBuilder();
		for (int i = trainSize; i < featureStrings.size(); i++){
			if (labels.get(i) >1 ){
				sb.append(labels.get(i) == 2?1:-1);
				sb.append(featureStrings.get(i));
				sb.append("\n");
			}
		}
		FileUtil.writeTextToFile(sb.toString(), "test.role.svm");

		float thresh1 = BioNLPSettings.getFloatValue("LINK_THRESHOLD"), thresh2 = BioNLPSettings.getFloatValue("ROLE_THRESHOLD");
		String[] predictions = FileUtil.getTextFromFile("svm_test_edge_predictions").split("\n");
		int[] predLabels = new int[predictions.length];
		// link scores and role scores
		float[] lscores = new float[predictions.length], rscores = new float[predictions.length]; 
		sb = new StringBuilder();
		for (int i = 0; i < predictions.length; i++){
			lscores[i] = Float.parseFloat(predictions[i]);
			if (lscores[i] > thresh1){
				if (labels.get(i+trainSize) == 1)
					predLabels[i] = 4;// either 2 or 3 can do
				else{
					predLabels[i] = -1;
					sb.append(labels.get(i+trainSize)==2?1:-1);
					sb.append(featureStrings.get(i+trainSize));
					sb.append("\n");
				}
			}
			else{
				// no link
				predLabels[i] = 1;
			}
		}
		FileUtil.writeTextToFile(sb.toString(), "test.role.pred.svm");
		System.err.println("[EDGE_LEARN] training role model");
		if (!new File(BioNLPSettings.getValue("ROLE_MODEL")).exists())
			svm.train("train.role.svm", BioNLPSettings.getValue("ROLE_MODEL")/*"svm_role_model"*/);
		else{
			System.err.println("[EDGE_LEARN] training role model skipped");
		}
//		svm.train("train.role.svm", BioNLPSettings.getValue("ROLE_MODEL")/*"svm_role_model"*/);
		System.err.println("[EDGE_LEARN] testing role model");
		svm.test("test.role.pred.svm", BioNLPSettings.getValue("ROLE_MODEL"), "svm_test_role_pred_predictions");

		int idx = 0;
		predictions = FileUtil.getTextFromFile("svm_test_role_pred_predictions").split("\n");
		for(int i = 0; i < predictions.length; i++){
			// find the first prediction index
			while (predLabels[idx] != -1){
				idx++;
			}
			rscores[idx] = Float.parseFloat(predictions[i]);
			if (rscores[idx] > thresh2){
				predLabels[idx] =  2;
			}
			else{
				predLabels[idx] =  3;
			}
			
		}

		// do the accuracy
		int cor = 0, pred = 0, truth = 0;
		int[][] confusion = new int[4][4];

		
		/*StringBuilder details = new StringBuilder();
		// ***get the edge/role model weights***
		String[] lines = FileUtil.getTextFromFile("svm_role_model2.weights").split("\n"); 
		String[] featureNames = new String[lines.length+1];
		float[] weights1 = new float[lines.length+1], weights2 = new float[lines.length+1];
		
		for (int i = 0; i < lines.length; i++){
			String[] items = lines[i].split("\t");
			weights2[Integer.parseInt(items[1])] = Float.parseFloat(items[2]);
			featureNames[Integer.parseInt(items[1])] = items[0];
		}
		lines = FileUtil.getTextFromFile("svm_edge_model2.weights").split("\n");
		for (int i = 0; i < lines.length; i++){
			String[] items = lines[i].split("\t");
			weights1[Integer.parseInt(items[1])] = Float.parseFloat(items[2]);
			featureNames[Integer.parseInt(items[1])] = items[0];
		}
		// ***END***
*/		
		for (int i = trainSize; i < labels.size(); i++){
			/*details.append("["+pairIds.get(i)+"] truth = "+labels.get(i) +", pred = "+predLabels[i-trainSize]+", linkscore = "
					+lscores[i-trainSize] +", rolescore = "+rscores[i-trainSize]+ ", features = {");
			String[] fea = featureStrings.get(i).split(" ");
			for (int j = 1; j < fea.length; j++){
				String[] pair = fea[j].split(":");
//				if (pair[0].equals(" ")){
//					System.err.println(featureStrings.get(i)+"@@"+fea[j]+",, "+j);
//				}
				details.append(" "+pair[0]+"("+featureNames[Integer.parseInt(pair[0])]+"):"
						+weights1[Integer.parseInt(pair[0])]+";"+weights2[Integer.parseInt(pair[0])]);
			}
			details.append("}\n");*/
			confusion[labels.get(i)-1][predLabels[i-trainSize]-1]++;
			if (labels.get(i )>1){
				truth++;
				if (predLabels[i- trainSize] >1){
					cor++;
				}
			}
			if (predLabels[i- trainSize] >1){
				pred++;
			}
		}

		
		float prec = (float) cor / pred, rec = (float)cor/truth;
		float f1 = 2/(1/prec+1/rec);
		String[] classes = new String[]{"NA","Theme","Cause", "LINK"};
		System.err.println("T\\P"+"\t"+classes[0]+"\t"+classes[1]+"\t"+classes[2]+"\t"+classes[3]);
		for (int i = 0; i < 3; i++){
			System.err.println(classes[i]+"\t"+confusion[i][0]+"\t"+confusion[i][1]+"\t"+confusion[i][2]+"\t"+confusion[i][3]);
		}
		System.err.println("precision = "+prec);
		System.err.println("recall = "+rec);
		System.err.println("f1 = "+f1);
		
//		System.err.println("********details********");
//		System.err.println(details.toString());
	}

}
